
# Handles cross validation data subset selection and picking the best parameter
# Params:
#   params: a vector of different parameter values to consider
#   evaluate: a function from (parameter,traindata,trainclass,testdata,testclass) to loss value
#   X: all of the training data (where X[,i] is the ith sample)
#   Y: all of the test data
# Returns:
#   a list L, where:
#     L$best = best parameter
#     L$errorRateVec = the error rate for each parameter
chooseParam <- function(params,evaluate,X,Y) {
  k <- floor(dim(X)[2]*0.2)
  vals <- c()
  for(i in 1:length(params)) {
    print(paste("crossvalidation round",i,"of",length(params)))
    test <- sample(1:(dim(X)[2]),k)
    train <- (1:(dim(X)[2]))[-test]
    
    #TODO: change testx and testy back
    vals[i] <- evaluate(params[i],X[,train],Y[,train],X[,test],Y[,test])
    print(sprintf("Cost when param = %d is %f", params[i], vals[i]))
  }
  return(list(best=params[which.min(vals)], costVec=vals))
}

#to evaluate expected accuracy, run evaluate(param,trnx,trny,tstx,tsty)

#evaluates the performance of a model run system.time(code that you want to time)

#takes learn:(trnx,trny,param) -> model and predict:(testx,model) -> prediction
getEvaluator <- function(learn,predict) {
  function(param,trnx,trny,testx,testy) {
    print("evaluationg")
    model <- learn(trnx,trny,param)
    print("learned, now evaluationing")
    error <- 0
    for(i in 1:dim(testx)[2]) {
#       print("type of predict(testx[,i]),model")
#       print(typeof(predict(testx[,i],model)))
#       print("type of testy[,i]")
#       print(typeof(testy[,i]))
      t <- predict(testx[,i],model)
      if(sum((t - testy[,i])^2) == 2) {
        error <- error + 1
      }
    }
    return ((error)/NCOL(testy))
  }
}

# measures the elapsed time spent on learning a model
getLearnTimeEvaluator <- function(learn) {
  function(param,trnx,trny,testx,testy) {
    print("evaluating time for learn")
    learnElapsedTime = system.time(model <- learn(trnx,trny,param))[3]
    
    return (learnElapsedTime)
  }
}

# measures the elapsed time spent on predicting labels of all the test points
getPredictTimeEvaluator <- function(learn, predict) {
  function(param,trnx,trny,testx,testy) {
    print("evaluating time for predict")
    model <- learn(trnx,trny,param)
    print("Learned model. Start predicting")
    #predictElapsedTime = system.time(
    ptm <- proc.time()
      error <- 0
      for(i in 1:dim(testx)[2]) {
        #       print("type of predict(testx[,i]),model")
        #       print(typeof(predict(testx[,i],model)))
        #       print("type of testy[,i]")
        #       print(typeof(testy[,i]))
        if(sum((predict(testx[,i],model) - testy[,i])^2) == 2) {
          error <- error + 1
        }
      }
    #)
    return (proc.time() - ptm)
  }
}

getCombinedTimeEvaluator <- function(learn, predict) {
  function(param,trnx,trny,testx,testy) {
    print("evaluating time for learn + predict")
    ptm <- proc.time()
    model <- learn(trnx,trny,param)
    print("Learned model. Start predicting")
    #predictElapsedTime = system.time(    
    error <- 0
    for(i in 1:dim(testx)[2]) {
#       print("type of predict(testx[,i]),model")
#       print(typeof(predict(testx[,i],model)))
#       print("length of predict(testx[,i]),model")
#       print(length(predict(testx[,i],model)))
#       print(predict(testx[,i],model))
#       print("type of testy[,i]")
#       print(typeof(testy[,i]))
#       print("length of predict(testx[,i]),model")
#       print(length(testy[,i]))
#       print(testy[,i])
      if(sum((predict(testx[,i],model) - testy[,i])^2) == 2) {
        error <- error + 1
      }
    }
    #)
    return (proc.time() - ptm)
  }
}
